setwd("")

rm(list=ls())

library(dplyr)
library(tidyr)
library(cocor)
library(stringr)
library(xtable)
source("fun.R")


load("MannheimPartyPositions.Rdata")


# Drop some observations that are not a 'close match'
d <- partypos %>% 
 mutate( eblrs = ifelse(year_diff_eb!=0, NA, eblrs), 
         cmlrs = ifelse(year_diff_cm>3, NA, cmlrs), 
         hilrs = ifelse(year_diff_hi>3, NA, hilrs) ) 

d <- d %>% 
    rename( w1=wordsc, w2=theta.het, bst=bstlr, eb=eblrs, bl=bllrs, cm=cmlrs, hi=hilrs) %>% 
    select(-year_diff_eb, -year_diff_cm, -year_diff_hi, -ebfeellrs, -theta.wf) 

dd <- d %>% gather(vmeasure, value, -country, -cmp_id, -year) 
d <- d %>% gather(vmeasure, value, -country, -cmp_id, -year, -rile, -mcsslrs) 

d <- d %>% group_by(country, vmeasure) %>%
    mutate(mcsslrs = mcsslrs/sd(mcsslrs,na.rm=TRUE),
          rile = rile/sd(rile,na.rm=TRUE), 
          value = value/sd(value, na.rm=TRUE),
          n = sum(!is.na(value)) ) 

d <- d %>% filter(n != 0) %>% group_by(country,vmeasure) %>% 
    filter( !is.na(value) ) %>% 
    summarize( r_hm = cor(mcsslrs, value, use="na.or.complete"), 
               r_hr = cor(rile, value, use="na.or.complete"), 
               r_rm = cor(mcsslrs, rile, use="na.or.complete"), 
               alt = ifelse(r_hm < 0 & r_hr < 0, "l", "g"), 
               passed = ifelse( ((r_hm > r_hr) & alt == "g") | ( (r_hm < r_hr) & alt == "l"), 1, 0), 
               n = unique(n) ) 

d <- d %>% rowwise() %>% 
      mutate(meng1992 =  cor_test(r_hm, r_hr, r_rm, n, 
        alternative=alt, test="meng1992")@meng1992$p.value ) 


# Summarize #
#############

d <- d %>% mutate(vmeasure_type = "",   
              vmeasure_type = ifelse(vmeasure=="w1" | vmeasure == "w2", 'text', vmeasure_type),
              vmeasure_type = ifelse(vmeasure=="bst" | vmeasure == "eb", 'mass', vmeasure_type), 
              vmeasure_type = ifelse(vmeasure=="bl" | vmeasure == "cm" | vmeasure == "hi", 'expert', vmeasure_type) )

d %>% group_by(vmeasure_type) %>% 
  summarize( rile_win = sum(passed==0), 
             mcss_win=sum(passed==1), 
             p=mcss_win/(rile_win+mcss_win) )

table(d$passed)/nrow(d)

dd_agg <- na.omit(dd) %>% group_by(vmeasure) %>% 
  summarize( mu=mean(value,na.rm=TRUE), 
            sd=sd(value,na.rm=TRUE), 
            n=n(), n_parties=n_distinct(cmp_id), 
            n_election=n_distinct(year) ) 

print(xtable(dd_agg, digits=c(0,0,2,2,0,0,0)), 
    include.rownames=FALSE, booktabs=TRUE)


# Print #
##########

d <- as.data.frame(d)
countries <- with(d, unique(country))
K <- with(d, n_distinct(country) )

cat("", file="./results/table_corr.tex")
cat( "\\begin{sidewaystable}
\\begin{center}
\\begin{tabular}{lllllllllllllll}
\\toprule %%%%%%%%%%%%%%%%%%%%%%%%%
& \\multicolumn{2}{c}{Wordscores} & \\multicolumn{2}{c}{Wordfish} &  \\multicolumn{2}{c}{NES} & 
\\multicolumn{2}{c}{Eurobarometer} & \\multicolumn{2}{c}{Benoit/Laver} & \\multicolumn{2}{c}{Castle/Mair} &
\\multicolumn{2}{c}{Huber/Inglehart}\\\\
\\cmidrule(lr){2-3} \\cmidrule(lr){4-5}  \\cmidrule(lr){6-7}  \\cmidrule(lr){8-9}   \\cmidrule(lr){10-11}   \\cmidrule(lr){12-13}   \\cmidrule(lr){14-15}  
Country & MCSS & RILE & MCSS & RILE & MCSS & RILE & MCSS & RILE & MCSS & RILE & MCSS & RILE & MCSS & RILE\\\\
\\midrule %%%%%%%%%%%%%%%%%%%%%%%%% \n"
     , append=F, file="./results/table_corr.tex")
for(i in 1:K){
  cat( simpleCap(countries[i]), " & ",
  with( subset(d, country==countries[i] & vmeasure == "w1"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "w1"),  printcorr(r_hr, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "w2"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "w2"),  printcorr(r_hr, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "bst"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "bst"),  printcorr(r_hr, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "eb"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "eb"),  printcorr(r_hr, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "bl"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "bl"),  printcorr(r_hr, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "cm"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "cm"),  printcorr(r_hr, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "hi"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "hi"),  printcorr(r_hr, passed) ) , "\\\\ \n", 
  sep="", file="./results/table_corr.tex", append=T )
}
cat("\\bottomrule %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\\end{tabular}
\\caption{}
\\end{center}
\\end{sidewaystable}"
    , append=T, file="./results/table_corr.tex")





cat("", file="./results/table_corr_A.tex")
cat( "\\begin{sidewaystable}
\\begin{center}
\\begin{tabular}{lllllllllllll}
\\toprule %%%%%%%%%%%%%%%%%%%%%%%%%
& \\multicolumn{3}{c}{Wordscores} & 
  \\multicolumn{3}{c}{Wordfish} &  
  \\multicolumn{3}{c}{NES} & 
  \\multicolumn{3}{c}{Eurobarometer} \\\\
\\cmidrule(lr){2-4} \\cmidrule(lr){5-7}  \\cmidrule(lr){8-10}  \\cmidrule(lr){11-13}  
Country & MCSS & RILE & $p$-value & MCSS & RILE & $p$-value & MCSS & RILE & $p$-value & MCSS & RILE & $p$-value \\\\
\\midrule %%%%%%%%%%%%%%%%%%%%%%%%% \n"
     , append=F, file="./results/table_corr_A.tex")
for(i in 1:K){
  cat( simpleCap(countries[i]), " & ",
  with( subset(d, country==countries[i] & vmeasure == "w1"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "w1"),  printcorr(r_hr, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "w1"),  printp(meng1992, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "w2"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "w2"),  printcorr(r_hr, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "w2"),  printp(meng1992, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "bst"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "bst"),  printcorr(r_hr, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "bst"),  printp(meng1992, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "eb"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "eb"),  printcorr(r_hr, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "eb"),  printp(meng1992, passed) ), "\\\\ \n", 
  sep="", file="./results/table_corr_A.tex", append=T )
}
cat("\\bottomrule %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\\end{tabular}
\\caption{ }
\\end{center}
\\end{sidewaystable}"
    , append=T, file="./results/table_corr_A.tex")




cat("", file="./results/table_corr_B.tex")
cat( "\\begin{sidewaystable}
\\begin{center}
\\begin{tabular}{llllllllll}
\\toprule %%%%%%%%%%%%%%%%%%%%%%%%%
& \\multicolumn{3}{c}{Benoit/Laver} & 
  \\multicolumn{3}{c}{Castle/Mair} &  
  \\multicolumn{3}{c}{Huber/Inglehart} \\\\
\\cmidrule(lr){2-4} \\cmidrule(lr){5-7}  \\cmidrule(lr){8-10} 
Country & MCSS & RILE & $p$-value & MCSS & RILE & $p$-value & MCSS & RILE & $p$-value \\\\
\\midrule %%%%%%%%%%%%%%%%%%%%%%%%% \n"
     , append=F, file="./results/table_corr_B.tex")
for(i in 1:K){
  cat( simpleCap(countries[i]), " & ",
  with( subset(d, country==countries[i] & vmeasure == "bl"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "bl"),  printcorr(r_hr, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "bl"),  printp(meng1992, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "cm"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "cm"),  printcorr(r_hr, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "cm"),  printp(meng1992, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "hi"),  printcorr(r_hm, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "hi"),  printcorr(r_hr, passed) ) , " & ", 
  with( subset(d, country==countries[i] & vmeasure == "hi"),  printp(meng1992, passed) ) , "\\\\ \n", 
  sep="", file="./results/table_corr_B.tex", append=T )
}
cat("\\bottomrule %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\\end{tabular}
\\caption{ }
\\end{center}
\\end{sidewaystable}"
    , append=T, file="./results/table_corr_B.tex")


